import requests
from bs4 import BeautifulSoup
import os
import re

# 获取要爬取网站的全部页面内容
def getHtmlUrl(url):
    try:
        rsp=requests.get(url)
        rsp.raise_for_status()
        rsp.encoding=rsp.apparent_encoding
        #返回网页所有内容
        return rsp.text
    except:
        print("获取页面全部内容异常，请检测访问地址是否异常")
        return "error"
# 设置需要访问图片的URL并放入到解析器中
def getImgUrl(img):
    href=img['href']
    html=getHtmlUrl(href)
    return html
# 设置需要访问图片的URL并放入到解析器中
def getBigImgUrl(img):
    href=img['href']
    bigUrl="http://www.netbian.com"+href
    text=getHtmlUrl(bigUrl)
    return text

#根据html解析完成的内容进行过滤并获取相关图片
def getJPG(html):
    soup = BeautifulSoup(html, 'html.parser')
    allImg = soup.find('div', class_ = 'list').find('ul').find_all("img")
    for img in allImg:
        imgName = img['alt']
        imgUrl=img['src']
        print("图片名称："+imgName+" 图片位置："+imgUrl)
        Imgpath = 'Img/' + imgName+'.jpg'
        try:
            if not os.path.exists('Img/'):
                os.mkdir('Img/')
            if not os.path.exists(Imgpath):
                r = requests.get(imgUrl)
                with open(Imgpath,'wb') as f:
                    f.write(r.content)
                    f.close()
                    print(imgName+" 下载成功")
            else:
                print(imgName+"图片已存在")
        except Exception as e:
            print("抓取图片失败，原因为："+str(e))

#获取放大图片的访问地址及页面内容
def getHtmlImgUrl(html):
    soup = BeautifulSoup(html, 'html.parser')
    #通过正则表达式，获取所要展示的图片的地址
    all_bigImg_html = soup.find('div', class_ = 'list').find('ul').find_all('a',attrs={'href':re.compile('^((?!http).)*$'),'target':'_blank'})
    for img in all_bigImg_html:
        imgName=img['title']
        imgUrl=img['href']
        print("图片大标题："+imgName+" 图片访问页面网址："+imgUrl)
        print("大图片界面的内容：")
        bigtext=getBigImgUrl(img)
        print(bigtext)

if __name__== '__main__':
    url='http://www.netbian.com/weimei/index.htm'
    text=getHtmlUrl(url)
    # getJPG(text)
    getHtmlImgUrl(text)
